{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 参考官网理解并实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先导入一些用到的库。\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-164ff3e31ae0>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_dataset/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_dataset/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./mnist_dataset/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./mnist_dataset/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "(55000, 784)\n",
      "(55000, 10)\n",
      "(5000, 784)\n",
      "(5000, 10)\n",
      "(10000, 784)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# 先来看看数据长什么样子\n",
    "mnist = input_data.read_data_sets(\"./mnist_dataset\",one_hot=True)\n",
    "\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "\n",
    "print(mnist.validation.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "\n",
    "print(mnist.test.images.shape)\n",
    "print(mnist.test.labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里，我们使用更加方便的InteractiveSession类。通过它，你可以更加灵活地构建你的代码。它能让你在运行图的时候，插入一些计算图，这些计算图是由某些操作(operations)构成的。这对于工作在交互式环境中的人们来说非常便利，比如使用IPython。如果你没有使用InteractiveSession，那么你需要在启动session之前构建整个计算图，然后启动该计算图。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多层卷积网络的softmax回归模型。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里的x和y并不是特定的值，相反，他们都只是一个占位符，可以在TensorFlow运行某一计算时根据该占位符输入具体的值。\n",
    "\n",
    "输入图片x是一个2维的浮点数张量。这里，分配给它的shape为[None, 784]，其中784是一张展平的MNIST图片的维度。None表示其值大小不定，在这里作为第一个维度值，用以指代batch的大小，意即x的数量不定。输出类别值y_也是一个2维张量，其中每一行为一个10维的one-hot向量,用于代表对应某一MNIST图片的类别。\n",
    "\n",
    "虽然placeholder的shape参数是可选的，但有了它，TensorFlow能够自动捕捉因数据维度不一致导致的错误。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Placeholder:0\", shape=(?, 784), dtype=float32) Tensor(\"Placeholder_1:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(\"float32\",shape=[None,784])  \n",
    "y_=tf.placeholder(\"float32\",shape=[None,10])  \n",
    "print(x, y_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们现在为模型定义权重W和偏置b。可以将它们当作额外的输入量，但是TensorFlow有一个更好的处理方式：变量。一个变量代表着TensorFlow计算图中的一个值，能够在计算过程中使用，甚至进行修改。在机器学习的应用过程中，模型参数一般用Variable来表示。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "W = tf.Variable(tf.zeros([784,10]))\n",
    "b = tf.Variable(tf.zeros([10]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在调用tf.Variable的时候传入初始值。在这个例子里，我们把W和b都初始化为零向量。W是一个784x10的矩阵（因为我们有784个特征和10个输出值）。b是一个10维的向量（因为我们有10个分类）。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "变量需要通过seesion初始化后，才能在session中使用。这一初始化步骤为，为初始值指定具体值（本例当中是全为零），并将其分配给每个变量,可以一次性为所有变量完成此操作。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 类别预测与损失函数\n",
    "现在我们可以实现我们的回归模型了。这只需要一行！我们把向量化后的图片x和权重矩阵W相乘，加上偏置b，然后计算每个分类的softmax概率值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = tf.nn.softmax(tf.matmul(x,W) + b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以很容易的为训练过程指定最小化误差用的损失函数，我们的损失函数是目标类别和预测类别之间的交叉熵。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意，tf.reduce_sum把minibatch里的每张图片的交叉熵值都加起来了。我们计算的交叉熵是指整个minibatch的。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型\n",
    "我们已经定义好模型和训练用的损失函数，那么用TensorFlow进行训练就很简单了。因为TensorFlow知道整个计算图，它可以使用自动微分法找到对于各个变量的损失的梯度值。TensorFlow有大量内置的优化算法 这个例子中，我们用最速下降法让交叉熵下降，步长为0.01."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这一行代码实际上是用来往计算图上添加一个新操作，其中包括计算梯度，计算每个参数的步长变化，并且计算出新的参数值。\n",
    "\n",
    "返回的train_step操作对象，在运行时会使用梯度下降来更新参数。因此，整个模型的训练可以通过反复地运行train_step来完成。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1000):\n",
    "  xs, ys = mnist.train.next_batch(50)\n",
    "  train_step.run(feed_dict={x: xs, y_: ys})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建一个多层卷积网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "权重初始化\n",
    "\n",
    "为了创建这个模型，我们需要创建大量的权重和偏置项。这个模型中的权重在初始化时应该加入少量的噪声来打破对称性以及避免0梯度。由于我们使用的是ReLU神经元，因此比较好的做法是用一个较小的正数来初始化偏置项，以避免神经元节点输出恒为0的问题（dead neurons）。为了不在建立模型的时候反复做初始化操作，我们定义两个函数用于初始化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 卷积和池化\n",
    "TensorFlow在卷积和池化上有很强的灵活性。我们怎么处理边界？步长应该设多大？在这个实例里，我们会一直使用vanilla版本。我们的卷积使用1步长（stride size），0边距（padding size）的模板，保证输出和输入是同一个大小。我们的池化用简单传统的2x2大小的模板做max pooling。为了代码更简洁，我们把这部分抽象成一个函数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第一层卷积\n",
    "现在我们可以开始实现第一层了。它由一个卷积接一个max pooling完成。卷积在每个5x5的patch中算出32个特征。卷积的权重张量形状是[5, 5, 1, 32]，前两个维度是patch的大小，接着是输入的通道数目，最后是输出的通道数目。 而对于每一个输出通道都有一个对应的偏置量。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "b_conv1 = bias_variable([32])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "为了用这一层，我们把x变成一个4d向量，其第2、第3维对应图片的宽、高，最后一维代表图片的颜色通道数(因为是灰度图所以这里的通道数为1，如果是rgb彩色图，则为3)。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_image = tf.reshape(x, [-1,28,28,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们把x_image和权值向量进行卷积，加上偏置项，然后应用ReLU激活函数，最后进行max pooling。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第二层卷积\n",
    "为了构建一个更深的网络，我们会把几个类似的层堆叠起来。第二层中，每个5x5的patch会得到64个特征。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "b_conv2 = bias_variable([64])\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "密集连接层\n",
    "\n",
    "现在，图片尺寸减小到7x7，我们加入一个有1024个神经元的全连接层，用于处理整个图片。我们把池化层输出的张量reshape成一些向量，乘上权重矩阵，加上偏置，然后对其使用ReLU。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "b_fc1 = bias_variable([1024])\n",
    "\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout\n",
    "\n",
    "为了减少过拟合，我们在输出层之前加入dropout。我们用一个placeholder来代表一个神经元的输出在dropout中保持不变的概率。这样我们可以在训练过程中启用dropout，在测试过程中关闭dropout。 TensorFlow的tf.nn.dropout操作除了可以屏蔽神经元的输出外，还会自动处理神经元输出值的scale。所以用dropout的时候可以不用考虑scale。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-23-0eb814f49c23>:2: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "keep_prob = tf.placeholder(\"float\")\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输出层\n",
    "最后，我们添加一个softmax层，就像前面的单层softmax regression一样。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_fc2 = weight_variable([1024, 10])\n",
    "b_fc2 = bias_variable([10])\n",
    "\n",
    "y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0 training accuracy 0.12\n",
      "step 100 training accuracy 0.88\n",
      "step 200 training accuracy 0.92\n",
      "step 300 training accuracy 0.94\n",
      "step 400 training accuracy 0.96\n",
      "step 500 training accuracy 0.96\n",
      "step 600 training accuracy 0.92\n",
      "step 700 training accuracy 0.96\n",
      "step 800 training accuracy 0.98\n",
      "step 900 training accuracy 1.0\n",
      "step 1000 training accuracy 0.94\n",
      "step 1100 training accuracy 0.92\n",
      "step 1200 training accuracy 1.0\n",
      "step 1300 training accuracy 0.94\n",
      "step 1400 training accuracy 0.98\n",
      "step 1500 training accuracy 0.98\n",
      "step 1600 training accuracy 0.98\n",
      "step 1700 training accuracy 1.0\n",
      "step 1800 training accuracy 0.96\n",
      "step 1900 training accuracy 1.0\n",
      "step 2000 training accuracy 0.98\n",
      "step 2100 training accuracy 0.98\n",
      "step 2200 training accuracy 0.98\n",
      "step 2300 training accuracy 1.0\n",
      "step 2400 training accuracy 1.0\n",
      "step 2500 training accuracy 1.0\n",
      "step 2600 training accuracy 1.0\n",
      "step 2700 training accuracy 0.98\n",
      "step 2800 training accuracy 1.0\n",
      "step 2900 training accuracy 0.96\n",
      "step 3000 training accuracy 1.0\n",
      "step 3100 training accuracy 0.94\n",
      "step 3200 training accuracy 0.96\n",
      "step 3300 training accuracy 1.0\n",
      "step 3400 training accuracy 0.98\n",
      "step 3500 training accuracy 1.0\n",
      "step 3600 training accuracy 1.0\n",
      "step 3700 training accuracy 0.98\n",
      "step 3800 training accuracy 1.0\n",
      "step 3900 training accuracy 0.94\n",
      "step 4000 training accuracy 0.98\n",
      "step 4100 training accuracy 0.96\n",
      "step 4200 training accuracy 1.0\n",
      "step 4300 training accuracy 0.98\n",
      "step 4400 training accuracy 1.0\n",
      "step 4500 training accuracy 1.0\n",
      "step 4600 training accuracy 1.0\n",
      "step 4700 training accuracy 0.96\n",
      "step 4800 training accuracy 0.98\n",
      "step 4900 training accuracy 0.98\n",
      "step 5000 training accuracy 1.0\n",
      "step 5100 training accuracy 1.0\n",
      "step 5200 training accuracy 1.0\n",
      "step 5300 training accuracy 0.98\n",
      "step 5400 training accuracy 1.0\n",
      "step 5500 training accuracy 1.0\n",
      "step 5600 training accuracy 1.0\n",
      "step 5700 training accuracy 1.0\n",
      "step 5800 training accuracy 0.98\n",
      "step 5900 training accuracy 1.0\n",
      "step 6000 training accuracy 1.0\n",
      "step 6100 training accuracy 0.98\n",
      "step 6200 training accuracy 1.0\n",
      "step 6300 training accuracy 1.0\n",
      "step 6400 training accuracy 1.0\n",
      "step 6500 training accuracy 1.0\n",
      "step 6600 training accuracy 1.0\n",
      "step 6700 training accuracy 0.96\n",
      "step 6800 training accuracy 1.0\n",
      "step 6900 training accuracy 0.98\n",
      "step 7000 training accuracy 0.98\n",
      "step 7100 training accuracy 1.0\n",
      "step 7200 training accuracy 1.0\n",
      "step 7300 training accuracy 1.0\n",
      "step 7400 training accuracy 1.0\n",
      "step 7500 training accuracy 1.0\n",
      "step 7600 training accuracy 1.0\n",
      "step 7700 training accuracy 1.0\n",
      "step 7800 training accuracy 1.0\n",
      "step 7900 training accuracy 1.0\n",
      "step 8000 training accuracy 1.0\n",
      "step 8100 training accuracy 1.0\n",
      "step 8200 training accuracy 1.0\n",
      "step 8300 training accuracy 1.0\n",
      "step 8400 training accuracy 0.98\n",
      "step 8500 training accuracy 1.0\n",
      "step 8600 training accuracy 1.0\n",
      "step 8700 training accuracy 1.0\n",
      "step 8800 training accuracy 1.0\n",
      "step 8900 training accuracy 1.0\n",
      "step 9000 training accuracy 1.0\n",
      "step 9100 training accuracy 0.98\n",
      "step 9200 training accuracy 1.0\n",
      "step 9300 training accuracy 1.0\n",
      "step 9400 training accuracy 1.0\n",
      "step 9500 training accuracy 1.0\n",
      "step 9600 training accuracy 1.0\n",
      "step 9700 training accuracy 0.98\n",
      "step 9800 training accuracy 1.0\n",
      "step 9900 training accuracy 1.0\n",
      "step 10000 training accuracy 1.0\n",
      "step 10100 training accuracy 1.0\n",
      "step 10200 training accuracy 1.0\n",
      "step 10300 training accuracy 1.0\n",
      "step 10400 training accuracy 1.0\n",
      "step 10500 training accuracy 1.0\n",
      "step 10600 training accuracy 1.0\n",
      "step 10700 training accuracy 1.0\n",
      "step 10800 training accuracy 1.0\n",
      "step 10900 training accuracy 1.0\n",
      "step 11000 training accuracy 1.0\n",
      "step 11100 training accuracy 1.0\n",
      "step 11200 training accuracy 0.98\n",
      "step 11300 training accuracy 1.0\n",
      "step 11400 training accuracy 1.0\n",
      "step 11500 training accuracy 1.0\n",
      "step 11600 training accuracy 1.0\n",
      "step 11700 training accuracy 1.0\n",
      "step 11800 training accuracy 1.0\n",
      "step 11900 training accuracy 1.0\n",
      "step 12000 training accuracy 0.98\n",
      "step 12100 training accuracy 1.0\n",
      "step 12200 training accuracy 1.0\n",
      "step 12300 training accuracy 1.0\n",
      "step 12400 training accuracy 1.0\n",
      "step 12500 training accuracy 1.0\n",
      "step 12600 training accuracy 1.0\n",
      "step 12700 training accuracy 1.0\n",
      "step 12800 training accuracy 1.0\n",
      "step 12900 training accuracy 1.0\n",
      "step 13000 training accuracy 1.0\n",
      "step 13100 training accuracy 1.0\n",
      "step 13200 training accuracy 1.0\n",
      "step 13300 training accuracy 1.0\n",
      "step 13400 training accuracy 1.0\n",
      "step 13500 training accuracy 1.0\n",
      "step 13600 training accuracy 1.0\n",
      "step 13700 training accuracy 1.0\n",
      "step 13800 training accuracy 1.0\n",
      "step 13900 training accuracy 1.0\n",
      "step 14000 training accuracy 1.0\n",
      "step 14100 training accuracy 1.0\n",
      "step 14200 training accuracy 1.0\n",
      "step 14300 training accuracy 1.0\n",
      "step 14400 training accuracy 1.0\n",
      "step 14500 training accuracy 1.0\n",
      "step 14600 training accuracy 1.0\n",
      "step 14700 training accuracy 1.0\n",
      "step 14800 training accuracy 1.0\n",
      "step 14900 training accuracy 1.0\n",
      "step 15000 training accuracy 1.0\n",
      "step 15100 training accuracy 1.0\n",
      "step 15200 training accuracy 1.0\n",
      "step 15300 training accuracy 1.0\n",
      "step 15400 training accuracy 1.0\n",
      "step 15500 training accuracy 1.0\n",
      "step 15600 training accuracy 1.0\n",
      "step 15700 training accuracy 1.0\n",
      "step 15800 training accuracy 1.0\n",
      "step 15900 training accuracy 1.0\n",
      "step 16000 training accuracy 1.0\n",
      "step 16100 training accuracy 1.0\n",
      "step 16200 training accuracy 1.0\n",
      "step 16300 training accuracy 1.0\n",
      "step 16400 training accuracy 0.98\n",
      "step 16500 training accuracy 1.0\n",
      "step 16600 training accuracy 1.0\n",
      "step 16700 training accuracy 1.0\n",
      "step 16800 training accuracy 1.0\n",
      "step 16900 training accuracy 1.0\n",
      "step 17000 training accuracy 1.0\n",
      "step 17100 training accuracy 1.0\n",
      "step 17200 training accuracy 1.0\n",
      "step 17300 training accuracy 1.0\n",
      "step 17400 training accuracy 1.0\n",
      "step 17500 training accuracy 1.0\n",
      "step 17600 training accuracy 1.0\n",
      "step 17700 training accuracy 1.0\n",
      "step 17800 training accuracy 1.0\n",
      "step 17900 training accuracy 1.0\n",
      "step 18000 training accuracy 1.0\n",
      "step 18100 training accuracy 1.0\n",
      "step 18200 training accuracy 1.0\n",
      "step 18300 training accuracy 1.0\n",
      "step 18400 training accuracy 1.0\n",
      "step 18500 training accuracy 1.0\n",
      "step 18600 training accuracy 1.0\n",
      "step 18700 training accuracy 1.0\n",
      "step 18800 training accuracy 1.0\n",
      "step 18900 training accuracy 1.0\n",
      "step 19000 training accuracy 1.0\n",
      "step 19100 training accuracy 1.0\n",
      "step 19200 training accuracy 1.0\n",
      "step 19300 training accuracy 1.0\n",
      "step 19400 training accuracy 1.0\n",
      "step 19500 training accuracy 1.0\n",
      "step 19600 training accuracy 1.0\n",
      "step 19700 training accuracy 1.0\n",
      "step 19800 training accuracy 1.0\n",
      "step 19900 training accuracy 1.0\n",
      "test accuracy 0.9911\n"
     ]
    }
   ],
   "source": [
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))\n",
    "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "sess.run(tf.global_variables_initializer())\n",
    "for i in range(20000):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            x:batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "        print(\"step\", i, \"training accuracy\", train_accuracy)\n",
    "    train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})\n",
    "\n",
    "acc = accuracy.eval(feed_dict={\n",
    "        x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})\n",
    "print(\"test accuracy\", acc)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
